{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import argparse\n",
    "import sys\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "\n",
    "FLAGS = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting ./input_data\\train-images-idx3-ubyte.gz\n",
      "Extracting ./input_data\\train-labels-idx1-ubyte.gz\n",
      "Extracting ./input_data\\t10k-images-idx3-ubyte.gz\n",
      "Extracting ./input_data\\t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "# Import data\n",
    "data_dir = './input_data'\n",
    "mnist = input_data.read_data_sets(data_dir, one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "neuron = 784"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 输入层"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Create the model\n",
    "# 声明placeholder参数化的从外部输入数据，None代表维度可以是任意值\n",
    "x = tf.placeholder(tf.float32, [None, 784])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Ground Truth\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 隐层1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "random_seed = 0.33"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "W1 = tf.Variable(tf.truncated_normal([784, neuron_1], mean=0,stddev=0.1,seed=random_seed))\n",
    "b1 = tf.Variable(tf.truncated_normal([neuron_1], mean=0,stddev=0.1,seed=random_seed))\n",
    "logits1 = tf.matmul(x, W1) + b1\n",
    "y1 = tf.nn.relu(logits1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 隐层2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "W2 = tf.Variable(tf.truncated_normal([784, neuron_2], mean=0,stddev=0.1,seed=random_seed))\n",
    "b2 = tf.Variable(tf.truncated_normal([neuron_2], mean=0,stddev=0.1,seed=random_seed))\n",
    "logits2 = tf.matmul(x, W2) + b2\n",
    "y2 = tf.nn.relu(logits2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 输出层"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "W = tf.Variable(tf.zeros([neuron_2, 10]))\n",
    "b = tf.Variable(tf.zeros([10]))\n",
    "y = tf.matmul(y2, W) + b"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 计算交叉熵with正则项\n",
    "接下来我们计算交叉熵，注意这里不要使用注释中的手动计算方式，而是使用系统函数。\n",
    "另一个注意点就是，softmax_cross_entropy_with_logits的logits参数是**未经激活的wx+b**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# The raw formulation of cross-entropy,\n",
    "#\n",
    "#   tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)),\n",
    "#                                 reduction_indices=[1]))\n",
    "#\n",
    "# can be numerically unstable.\n",
    "#\n",
    "# So here we use tf.nn.softmax_cross_entropy_with_logits on the raw\n",
    "# outputs of 'y', and then average across the batch.\n",
    "reg_lambda = 0.01 #惩罚因子，值越大，惩罚力度越大\n",
    "regularizer = tf.contrib.layers.l2_regularizer(reg_lambda)\n",
    "regularization = regularizer(W) +  regularizer(b) \n",
    "\n",
    "cross_entropy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y)\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "验证我们模型在测试数据上的准确率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def fn(learning_rate, iteration, num_batch):\n",
    "    # 梯度下降，学习率为0.3，并使用交叉熵优化\n",
    "    train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(cross_entropy)\n",
    "    sess = tf.Session()\n",
    "    init_op = tf.global_variables_initializer()\n",
    "    sess.run(init_op)\n",
    "    \n",
    "    # Train\n",
    "    for _ in range(iteration):\n",
    "        batch_xs, batch_ys = mnist.train.next_batch(num_batch)\n",
    "        sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "    # Test trained model\n",
    "    correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "    print(\"Accuracy: \", sess.run(accuracy, feed_dict={x: mnist.test.images, y_: mnist.test.labels}), \"| when lr = \", learning_rate, \"| iteration = \", iteration, \"| num_batch = \", num_batch, \"| neuron_1 = \", neuron_1, \"| neuron_2 = \", neuron_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lr_arr = [0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "iteration = 8000\n",
    "num_batch = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  0.9827 | when lr =  0.8 | iteration =  8000 | num_batch =  100 | neuron_1 =  784 | neuron_2 =  784\n",
      "Accuracy:  0.9825 | when lr =  0.7 | iteration =  8000 | num_batch =  100 | neuron_1 =  784 | neuron_2 =  784\n",
      "Accuracy:  0.9831 | when lr =  0.6 | iteration =  8000 | num_batch =  100 | neuron_1 =  784 | neuron_2 =  784\n",
      "Accuracy:  0.9818 | when lr =  0.5 | iteration =  8000 | num_batch =  100 | neuron_1 =  784 | neuron_2 =  784\n",
      "Accuracy:  0.9825 | when lr =  0.4 | iteration =  8000 | num_batch =  100 | neuron_1 =  784 | neuron_2 =  784\n",
      "Accuracy:  0.9812 | when lr =  0.3 | iteration =  8000 | num_batch =  100 | neuron_1 =  784 | neuron_2 =  784\n",
      "Accuracy:  0.9802 | when lr =  0.2 | iteration =  8000 | num_batch =  100 | neuron_1 =  784 | neuron_2 =  784\n"
     ]
    }
   ],
   "source": [
    "for lr in lr_arr:\n",
    "    fn(lr, iteration, num_batch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "iteration = 10000\n",
    "num_batch = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  0.9823 | when lr =  0.8 | iteration =  10000 | num_batch =  100 | neuron_1 =  784 | neuron_2 =  784\n",
      "Accuracy:  0.9831 | when lr =  0.7 | iteration =  10000 | num_batch =  100 | neuron_1 =  784 | neuron_2 =  784\n",
      "Accuracy:  0.9827 | when lr =  0.6 | iteration =  10000 | num_batch =  100 | neuron_1 =  784 | neuron_2 =  784\n",
      "Accuracy:  0.9823 | when lr =  0.5 | iteration =  10000 | num_batch =  100 | neuron_1 =  784 | neuron_2 =  784\n",
      "Accuracy:  0.9822 | when lr =  0.4 | iteration =  10000 | num_batch =  100 | neuron_1 =  784 | neuron_2 =  784\n",
      "Accuracy:  0.9822 | when lr =  0.3 | iteration =  10000 | num_batch =  100 | neuron_1 =  784 | neuron_2 =  784\n",
      "Accuracy:  0.9807 | when lr =  0.2 | iteration =  10000 | num_batch =  100 | neuron_1 =  784 | neuron_2 =  784\n"
     ]
    }
   ],
   "source": [
    "for lr in lr_arr:\n",
    "    fn(lr, iteration, num_batch)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 本次调优，采用了2个隐层，采用了学习率为0.31，迭代次数6000，配合一些优化，达到0.9805"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "毫无疑问，这个模型是一个非常简陋，性能也不理想的模型。目前只能达到92%左右的准确率。\n",
    "接下来，希望大家利用现有的知识，将这个模型优化至98%以上的准确率。\n",
    "Hint：\n",
    "- 多隐层\n",
    "- 激活函数\n",
    "- 正则化\n",
    "- 初始化\n",
    "- 摸索一下各个超参数\n",
    "  - 隐层神经元数量\n",
    "  - 学习率\n",
    "  - 正则化惩罚因子\n",
    "  - 最好每隔几个step就对loss、accuracy等等进行一次输出，这样才能有根据地进行调整"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
